/*      */ package com.zimbra.cs.index;
/*      */ 
/*      */ import com.google.common.base.Objects;
/*      */ import com.google.common.base.Objects.ToStringHelper;
/*      */ import com.google.common.cache.Cache;
/*      */ import com.google.common.cache.CacheBuilder;
/*      */ import com.google.common.cache.RemovalListener;
/*      */ import com.google.common.cache.RemovalNotification;
/*      */ import com.google.common.io.Closeables;
/*      */ import com.google.common.io.NullOutputStream;
/*      */ import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder;
/*      */ import com.googlecode.concurrentlinkedhashmap.EvictionListener;
/*      */ import com.zimbra.common.localconfig.KnownKey;
/*      */ import com.zimbra.common.localconfig.LC;
/*      */ import com.zimbra.common.service.ServiceException;
/*      */ import com.zimbra.common.util.Log;
/*      */ import com.zimbra.common.util.ZimbraLog;
/*      */ import com.zimbra.cs.mailbox.Folder;
/*      */ import com.zimbra.cs.mailbox.MailItem;
/*      */ import com.zimbra.cs.mailbox.Mailbox;
/*      */ import com.zimbra.cs.mailbox.MailboxIndex;
/*      */ import com.zimbra.cs.mailbox.MailboxIndex.IndexTask;
/*      */ import com.zimbra.cs.volume.Volume;
/*      */ import com.zimbra.cs.volume.VolumeManager;
/*      */ import java.io.Closeable;
/*      */ import java.io.File;
/*      */ import java.io.IOException;
/*      */ import java.io.PrintStream;
/*      */ import java.util.List;
/*      */ import java.util.NoSuchElementException;
/*      */ import java.util.concurrent.ConcurrentMap;
/*      */ import java.util.concurrent.RejectedExecutionException;
/*      */ import java.util.concurrent.Semaphore;
/*      */ import java.util.concurrent.TimeUnit;
/*      */ import java.util.concurrent.atomic.AtomicBoolean;
/*      */ import java.util.concurrent.atomic.AtomicInteger;
/*      */ import java.util.concurrent.locks.Condition;
/*      */ import java.util.concurrent.locks.Lock;
/*      */ import java.util.concurrent.locks.ReentrantLock;
/*      */ import org.apache.lucene.document.Document;
/*      */ import org.apache.lucene.index.CheckIndex;
/*      */ import org.apache.lucene.index.CheckIndex.Status;
/*      */ import org.apache.lucene.index.CorruptIndexException;
/*      */ import org.apache.lucene.index.IndexReader;
/*      */ import org.apache.lucene.index.IndexWriter;
/*      */ import org.apache.lucene.index.IndexWriterConfig;
/*      */ import org.apache.lucene.index.IndexWriterConfig.OpenMode;
/*      */ import org.apache.lucene.index.LogByteSizeMergePolicy;
/*      */ import org.apache.lucene.index.LogDocMergePolicy;
/*      */ import org.apache.lucene.index.SerialMergeScheduler;
/*      */ import org.apache.lucene.index.Term;
/*      */ import org.apache.lucene.index.TermEnum;
/*      */ import org.apache.lucene.search.BooleanQuery;
/*      */ import org.apache.lucene.search.IndexSearcher;
/*      */ import org.apache.lucene.search.Query;
/*      */ import org.apache.lucene.search.Sort;
/*      */ import org.apache.lucene.search.SortField;
/*      */ import org.apache.lucene.search.TermQuery;
/*      */ import org.apache.lucene.store.Directory;
/*      */ import org.apache.lucene.store.NoSuchDirectoryException;
/*      */ import org.apache.lucene.util.Version;
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ public final class LuceneIndex
/*      */   extends IndexStore
/*      */ {
/*   87 */   public static final Version VERSION = Version.LUCENE_24;
/*      */   
/*   89 */   private static final Semaphore READER_THROTTLE = new Semaphore(LC.zimbra_index_max_readers.intValue());
/*   90 */   private static final Semaphore WRITER_THROTTLE = new Semaphore(LC.zimbra_index_max_writers.intValue());
/*      */   
/*   92 */   private static final Cache<Integer, IndexSearcherImpl> SEARCHER_CACHE = CacheBuilder.newBuilder().maximumSize(LC.zimbra_index_reader_cache_size.intValue()).expireAfterAccess(LC.zimbra_index_reader_cache_ttl.intValue(), TimeUnit.SECONDS).removalListener(new RemovalListener()
/*      */   {
/*      */ 
/*      */ 
/*      */     public void onRemoval(RemovalNotification<Integer, LuceneIndex.IndexSearcherImpl> notification)
/*      */     {
/*      */ 
/*   99 */       Closeables.closeQuietly((Closeable)notification.getValue());
/*      */     }
/*   92 */   }).build();
/*      */   
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*  106 */   private static final ConcurrentMap<Integer, IndexSearcherImpl> GAL_SEARCHER_CACHE = new ConcurrentLinkedHashMap.Builder().maximumWeightedCapacity(LC.zimbra_galsync_index_reader_cache_size.intValue()).listener(new EvictionListener()
/*      */   {
/*      */ 
/*      */     public void onEviction(Integer mboxId, LuceneIndex.IndexSearcherImpl searcher)
/*      */     {
/*      */ 
/*  112 */       Closeables.closeQuietly(searcher);
/*      */     }
/*  106 */   }).build();
/*      */   
/*      */ 
/*      */ 
/*      */ 
/*      */   private final Mailbox mailbox;
/*      */   
/*      */ 
/*      */ 
/*      */   private final LuceneDirectory luceneDirectory;
/*      */   
/*      */ 
/*      */ 
/*  119 */   private final AtomicBoolean pendingDelete = new AtomicBoolean(false);
/*  120 */   private final WriterInfo writerInfo = new WriterInfo(null);
/*      */   
/*      */ 
/*      */ 
/*      */   private final class WriterInfo
/*      */   {
/*      */     private LuceneIndex.IndexWriterRef writerRef;
/*      */     
/*      */ 
/*  129 */     private final Lock lock = new ReentrantLock();
/*  130 */     private final Condition hasNoWriters = this.lock.newCondition();
/*      */     
/*      */     private WriterInfo() {
/*  133 */       this.writerRef = null;
/*      */     }
/*      */     
/*      */     public LuceneIndex.IndexWriterRef getWriterRef() {
/*  137 */       return this.writerRef;
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */     public void setWriterRef(LuceneIndex.IndexWriterRef newRef)
/*      */       throws IOException
/*      */     {
/*  145 */       if ((newRef != null) && (LuceneIndex.this.isPendingDelete())) {
/*  146 */         throw new IndexPendingDeleteException();
/*      */       }
/*  148 */       this.lock.lock();
/*      */       try {
/*  150 */         this.writerRef = newRef;
/*  151 */         if (this.writerRef == null) {
/*  152 */           this.hasNoWriters.signal();
/*      */         }
/*      */       } finally {
/*  155 */         this.lock.unlock();
/*      */       }
/*      */     }
/*      */     
/*      */     public Condition getHasNoWritersCondition() {
/*  160 */       return this.hasNoWriters;
/*      */     }
/*      */     
/*      */     public Lock getHasNoWritersLock() {
/*  164 */       return this.lock;
/*      */     }
/*      */   }
/*      */   
/*      */   private LuceneIndex(Mailbox mbox) throws ServiceException {
/*  169 */     this.mailbox = mbox;
/*  170 */     Volume vol = VolumeManager.getInstance().getVolume(mbox.getIndexVolume());
/*  171 */     String dir = vol.getMailboxDir(this.mailbox.getId(), (short)10);
/*      */     
/*      */ 
/*  174 */     File root = new File(dir + File.separatorChar + '0');
/*      */     
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*  183 */     if (!root.exists()) {
/*  184 */       root.mkdirs();
/*      */     }
/*      */     
/*  187 */     if (!root.canRead()) {
/*  188 */       throw ServiceException.FAILURE("LuceneDirectory not readable mbox=" + mbox.getId() + ",dir=" + root, null);
/*      */     }
/*  190 */     if (!root.canWrite()) {
/*  191 */       throw ServiceException.FAILURE("LuceneDirectory not writable mbox=" + mbox.getId() + ",dir=" + root, null);
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*  198 */     File segments = new File(root, "segments");
/*  199 */     if (!segments.exists()) {
/*  200 */       File newSegments = new File(root, "segments.new");
/*  201 */       if (newSegments.exists()) {
/*  202 */         newSegments.renameTo(segments);
/*      */       }
/*      */     }
/*      */     try
/*      */     {
/*  207 */       this.luceneDirectory = LuceneDirectory.open(root);
/*      */     } catch (IOException e) {
/*  209 */       throw ServiceException.FAILURE("Failed to create LuceneDirectory: " + root, e);
/*      */     }
/*      */   }
/*      */   
/*      */   public String toString()
/*      */   {
/*  215 */     return Objects.toStringHelper(this).add("mbox", this.mailbox.getId()).add("dir", this.luceneDirectory).toString();
/*      */   }
/*      */   
/*      */   private synchronized void doDeleteIndex() throws IOException {
/*  219 */     assert (this.writerInfo.getWriterRef() == null);
/*  220 */     ZimbraLog.index.debug("Deleting index %s", new Object[] { this.luceneDirectory });
/*  221 */     if (this.mailbox.isGalSyncMailbox()) {
/*  222 */       Closeables.closeQuietly((Closeable)GAL_SEARCHER_CACHE.remove(Integer.valueOf(this.mailbox.getId())));
/*      */     } else {
/*  224 */       SEARCHER_CACHE.asMap().remove(Integer.valueOf(this.mailbox.getId()));
/*      */     }
/*      */     String[] files;
/*      */     try
/*      */     {
/*  229 */       files = this.luceneDirectory.listAll();
/*      */     } catch (NoSuchDirectoryException ignore) {
/*  231 */       return;
/*      */     } catch (IOException e) {
/*  233 */       ZimbraLog.index.warn("Failed to delete index: %s", this.luceneDirectory, e);
/*  234 */       return;
/*      */     }
/*      */     
/*  237 */     for (String file : files) {
/*  238 */       this.luceneDirectory.deleteFile(file);
/*      */     }
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */   public void deleteIndex()
/*      */     throws IOException
/*      */   {
/*  247 */     this.pendingDelete.set(true);
/*  248 */     this.writerInfo.getHasNoWritersLock().lock();
/*      */     try {
/*  250 */       if (this.writerInfo.getWriterRef() != null) {
/*  251 */         this.writerInfo.getHasNoWritersCondition().awaitUninterruptibly();
/*      */       }
/*  253 */       doDeleteIndex();
/*      */     } finally {
/*  255 */       this.writerInfo.getHasNoWritersLock().unlock();
/*      */     }
/*  257 */     this.pendingDelete.set(false);
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */   public synchronized void warmup()
/*      */   {
/*  266 */     if ((SEARCHER_CACHE.asMap().containsKey(Integer.valueOf(this.mailbox.getId()))) || (GAL_SEARCHER_CACHE.containsKey(Integer.valueOf(this.mailbox.getId()))))
/*      */     {
/*  268 */       return;
/*      */     }
/*  270 */     long start = System.currentTimeMillis();
/*  271 */     IndexSearcher searcher = null;
/*      */     try {
/*  273 */       searcher = (IndexSearcher)openSearcher();
/*  274 */       searcher.search(new TermQuery(new Term("l.content", "zimbra")), 1, new Sort(new SortField("l.date", 3, true)));
/*      */     }
/*      */     catch (IOException e) {
/*  277 */       ZimbraLog.search.warn("Failed to warm up", e);
/*      */     } finally {
/*  279 */       Closeables.closeQuietly(searcher);
/*      */     }
/*  281 */     ZimbraLog.search.debug("WarmUpLuceneSearcher elapsed=%d", new Object[] { Long.valueOf(System.currentTimeMillis() - start) });
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */ 
/*      */   public void evict()
/*      */   {
/*  289 */     if (this.mailbox.isGalSyncMailbox()) {
/*  290 */       Closeables.closeQuietly((Closeable)GAL_SEARCHER_CACHE.remove(Integer.valueOf(this.mailbox.getId())));
/*      */     } else {
/*  292 */       SEARCHER_CACHE.asMap().remove(Integer.valueOf(this.mailbox.getId()));
/*      */     }
/*      */   }
/*      */   
/*      */   private IndexReader openIndexReader(boolean tryRepair) throws IOException {
/*      */     try {
/*  298 */       return IndexReader.open(this.luceneDirectory, null, true, LC.zimbra_index_lucene_term_index_divisor.intValue());
/*      */     } catch (CorruptIndexException e) {
/*  300 */       if (!tryRepair) {
/*  301 */         throw e;
/*      */       }
/*  303 */       repair(e);
/*  304 */       return openIndexReader(false);
/*      */     } catch (AssertionError e) {
/*  306 */       if (!tryRepair) {
/*  307 */         throw e;
/*      */       }
/*  309 */       repair(e); }
/*  310 */     return openIndexReader(false);
/*      */   }
/*      */   
/*      */   private IndexWriter openIndexWriter(IndexWriterConfig.OpenMode mode, boolean tryRepair) throws IOException
/*      */   {
/*      */     try {
/*  316 */       IndexWriter writer = new IndexWriter(this.luceneDirectory, getWriterConfig().setOpenMode(mode))
/*      */       {
/*      */ 
/*      */         public void message(String message)
/*      */         {
/*      */ 
/*  322 */           ZimbraLog.index.debug("IW: %s", new Object[] { message });
/*      */         }
/*      */       };
/*  325 */       if (ZimbraLog.index.isDebugEnabled())
/*      */       {
/*  327 */         writer.setInfoStream(new PrintStream(new NullOutputStream()));
/*      */       }
/*  329 */       return writer;
/*      */     } catch (AssertionError e) {
/*  331 */       unlockIndexWriter();
/*  332 */       if (!tryRepair) {
/*  333 */         throw e;
/*      */       }
/*  335 */       repair(e);
/*  336 */       return openIndexWriter(mode, false);
/*      */     } catch (CorruptIndexException e) {
/*  338 */       unlockIndexWriter();
/*  339 */       if (!tryRepair) {
/*  340 */         throw e;
/*      */       }
/*  342 */       repair(e); }
/*  343 */     return openIndexWriter(mode, false);
/*      */   }
/*      */   
/*      */   private synchronized <T extends Throwable> void repair(T ex) throws Throwable
/*      */   {
/*  348 */     ZimbraLog.index.error("Index corrupted", ex);
/*  349 */     LuceneIndexRepair repair = new LuceneIndexRepair(this.luceneDirectory);
/*      */     try {
/*  351 */       if (repair.repair() > 0) {
/*  352 */         ZimbraLog.index.info("Index repaired, re-indexing is recommended.");
/*      */       } else {
/*  354 */         ZimbraLog.index.warn("Unable to repair, re-indexing is required.");
/*  355 */         throw ex;
/*      */       }
/*      */     } catch (IOException e) {
/*  358 */       ZimbraLog.index.warn("Failed to repair, re-indexing is required.", e);
/*  359 */       throw ex;
/*      */     }
/*      */   }
/*      */   
/*      */   private void unlockIndexWriter() {
/*      */     try {
/*  365 */       IndexWriter.unlock(this.luceneDirectory);
/*      */     } catch (IOException e) {
/*  367 */       ZimbraLog.index.warn("Failed to unlock IndexWriter %s", this, e);
/*      */     }
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */   public synchronized ZimbraIndexSearcher openSearcher()
/*      */     throws IOException
/*      */   {
/*  379 */     IndexSearcherImpl searcher = null;
/*  380 */     if (this.mailbox.isGalSyncMailbox()) {
/*  381 */       searcher = (IndexSearcherImpl)GAL_SEARCHER_CACHE.get(Integer.valueOf(this.mailbox.getId()));
/*      */     } else {
/*  383 */       searcher = (IndexSearcherImpl)SEARCHER_CACHE.getIfPresent(Integer.valueOf(this.mailbox.getId()));
/*      */     }
/*  385 */     if (searcher != null) {
/*  386 */       ZimbraLog.search.debug("CacheHitLuceneSearcher %s", new Object[] { searcher });
/*  387 */       searcher.inc();
/*  388 */       return searcher;
/*      */     }
/*      */     
/*  391 */     READER_THROTTLE.acquireUninterruptibly();
/*  392 */     long start = System.currentTimeMillis();
/*      */     try {
/*  394 */       searcher = new IndexSearcherImpl(openIndexReader(true));
/*      */     }
/*      */     catch (IOException e)
/*      */     {
/*  398 */       if (isEmptyDirectory(this.luceneDirectory.getDirectory()))
/*      */       {
/*  400 */         IndexWriter writer = new IndexWriter(this.luceneDirectory, getWriterConfig().setOpenMode(IndexWriterConfig.OpenMode.CREATE));
/*      */         
/*  402 */         Closeables.closeQuietly(writer);
/*  403 */         searcher = new IndexSearcherImpl(openIndexReader(false));
/*      */       } else {
/*  405 */         throw e;
/*      */       }
/*      */     } finally {
/*  408 */       if (searcher == null) {
/*  409 */         READER_THROTTLE.release();
/*      */       }
/*      */     }
/*      */     
/*  413 */     ZimbraLog.search.debug("OpenLuceneSearcher %s,elapsed=%d", new Object[] { searcher, Long.valueOf(System.currentTimeMillis() - start) });
/*  414 */     searcher.inc();
/*  415 */     if (this.mailbox.isGalSyncMailbox()) {
/*  416 */       Closeables.closeQuietly((Closeable)GAL_SEARCHER_CACHE.put(Integer.valueOf(this.mailbox.getId()), searcher));
/*      */     } else {
/*  418 */       SEARCHER_CACHE.asMap().put(Integer.valueOf(this.mailbox.getId()), searcher);
/*      */     }
/*  420 */     return searcher;
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */   private boolean isEmptyDirectory(File dir)
/*      */   {
/*  431 */     if (!dir.exists()) {
/*  432 */       dir.mkdirs();
/*  433 */       return true;
/*      */     }
/*      */     
/*      */ 
/*  437 */     File[] files = dir.listFiles();
/*      */     
/*      */ 
/*  440 */     if (files == null) {
/*  441 */       ZimbraLog.index.warn("Could not list files in directory %s", new Object[] { dir.getAbsolutePath() });
/*  442 */       return false;
/*      */     }
/*      */     
/*  445 */     int num = 0;
/*  446 */     for (File file : files) {
/*  447 */       String fname = file.getName();
/*  448 */       if ((!file.isDirectory()) || ((!fname.equals(".")) && (!fname.equals(".."))))
/*      */       {
/*      */ 
/*  451 */         num++; }
/*      */     }
/*  453 */     return num <= 0;
/*      */   }
/*      */   
/*      */   public synchronized Indexer openIndexer() throws IOException
/*      */   {
/*  458 */     if (this.writerInfo.getWriterRef() != null) {
/*  459 */       this.writerInfo.getWriterRef().inc();
/*      */     } else {
/*  461 */       WRITER_THROTTLE.acquireUninterruptibly();
/*      */       try {
/*  463 */         this.writerInfo.setWriterRef(openWriter());
/*      */       } finally {
/*  465 */         if (this.writerInfo.getWriterRef() == null) {
/*  466 */           WRITER_THROTTLE.release();
/*      */         }
/*      */       }
/*      */     }
/*  470 */     return new LuceneIndexerImpl(this.writerInfo.getWriterRef());
/*      */   }
/*      */   
/*      */   private IndexWriterRef openWriter() throws IOException {
/*  474 */     assert (Thread.holdsLock(this));
/*      */     
/*      */     try
/*      */     {
/*  478 */       writer = openIndexWriter(IndexWriterConfig.OpenMode.APPEND, true);
/*      */     }
/*      */     catch (IOException e)
/*      */     {
/*      */       IndexWriter writer;
/*      */       
/*      */ 
/*  485 */       if (isEmptyDirectory(this.luceneDirectory.getDirectory())) {
/*  486 */         writer = openIndexWriter(IndexWriterConfig.OpenMode.CREATE, false);
/*      */       } else
/*  488 */         throw e;
/*      */     }
/*      */     IndexWriter writer;
/*  491 */     return new IndexWriterRef(this, writer);
/*      */   }
/*      */   
/*      */   private synchronized void commitWriter() throws IOException {
/*  495 */     assert (this.writerInfo.getWriterRef() != null);
/*      */     
/*  497 */     ZimbraLog.index.debug("Commit IndexWriter");
/*      */     
/*  499 */     MergeTask task = new MergeTask(this.writerInfo.getWriterRef());
/*      */     
/*  501 */     boolean success = false;
/*      */     try {
/*      */       try {
/*  504 */         this.writerInfo.getWriterRef().get().commit();
/*      */       } catch (CorruptIndexException e) {
/*      */         try {
/*  507 */           this.writerInfo.getWriterRef().get().close(false);
/*      */         }
/*      */         catch (Throwable ignore) {}
/*  510 */         repair(e);
/*  511 */         throw e;
/*      */       } catch (AssertionError e) {
/*      */         try {
/*  514 */           this.writerInfo.getWriterRef().get().close(false);
/*      */         }
/*      */         catch (Throwable ignore) {}
/*  517 */         this.writerInfo.getWriterRef().get().close(false);
/*  518 */         repair(e);
/*  519 */         throw e;
/*      */       }
/*  521 */       this.mailbox.index.submit(task);
/*  522 */       success = true;
/*      */     } catch (RejectedExecutionException e) {
/*  524 */       ZimbraLog.index.warn("Skipping merge because all index threads are busy");
/*      */     } finally {
/*  526 */       if (!success) {
/*  527 */         this.writerInfo.getWriterRef().dec();
/*      */       }
/*      */     }
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */   private synchronized void closeWriter()
/*      */   {
/*  536 */     if (this.writerInfo.getWriterRef() == null) {
/*  537 */       return;
/*      */     }
/*      */     
/*  540 */     ZimbraLog.index.debug("Close IndexWriter");
/*      */     try
/*      */     {
/*  543 */       this.writerInfo.getWriterRef().get().close(false); return;
/*      */     } catch (CorruptIndexException e) {
/*      */       try {
/*  546 */         repair(e);
/*      */       }
/*      */       catch (CorruptIndexException ignore) {}
/*      */     } catch (AssertionError e) {
/*  550 */       repair(e);
/*      */     } catch (IOException e) {
/*  552 */       ZimbraLog.index.error("Failed to close IndexWriter", e);
/*      */     } finally {
/*  554 */       unlockIndexWriter();
/*  555 */       WRITER_THROTTLE.release();
/*      */       try {
/*  557 */         this.writerInfo.setWriterRef(null);
/*      */       }
/*      */       catch (IOException e) {}
/*      */     }
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */   public boolean verify(PrintStream out)
/*      */     throws IOException
/*      */   {
/*  572 */     if (!IndexReader.indexExists(this.luceneDirectory)) {
/*  573 */       out.println("index does not exist or no segments file found: " + this.luceneDirectory.getDirectory());
/*  574 */       return true;
/*      */     }
/*  576 */     CheckIndex check = new CheckIndex(this.luceneDirectory);
/*  577 */     if (out != null) {
/*  578 */       check.setInfoStream(out);
/*      */     }
/*  580 */     CheckIndex.Status status = check.checkIndex();
/*  581 */     return status.clean;
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */   private static final class MergeScheduler
/*      */     extends SerialMergeScheduler
/*      */   {
/*  589 */     private final ReentrantLock lock = new ReentrantLock();
/*      */     
/*      */ 
/*      */ 
/*      */     void lock()
/*      */     {
/*  595 */       this.lock.lock();
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */     boolean tryLock()
/*      */     {
/*  604 */       return this.lock.tryLock();
/*      */     }
/*      */     
/*      */     void release() {
/*      */       try {
/*  609 */         this.lock.unlock();
/*      */       }
/*      */       catch (IllegalMonitorStateException ignore) {}
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */     public void merge(IndexWriter writer)
/*      */       throws CorruptIndexException, IOException
/*      */     {
/*  619 */       if (this.lock.isHeldByCurrentThread()) {
/*  620 */         super.merge(writer);
/*      */       }
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */ 
/*      */     public void close()
/*      */     {
/*  629 */       super.close();
/*  630 */       release();
/*      */     }
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */   private final class MergeTask
/*      */     extends MailboxIndex.IndexTask
/*      */   {
/*      */     private final LuceneIndex.IndexWriterRef ref;
/*      */     
/*      */ 
/*      */     MergeTask(LuceneIndex.IndexWriterRef ref)
/*      */     {
/*  644 */       super();
/*  645 */       this.ref = ref;
/*      */     }
/*      */     
/*      */     public void exec() throws IOException
/*      */     {
/*  650 */       IndexWriter writer = this.ref.get();
/*  651 */       LuceneIndex.MergeScheduler scheduler = (LuceneIndex.MergeScheduler)writer.getConfig().getMergeScheduler();
/*      */       try {
/*  653 */         if (scheduler.tryLock()) {
/*  654 */           writer.maybeMerge();
/*      */         } else {
/*  656 */           ZimbraLog.index.debug("Merge is in progress by other thread");
/*      */         }
/*      */       } catch (CorruptIndexException e) {
/*      */         try {
/*  660 */           writer.close(false);
/*      */         }
/*      */         catch (Throwable ignore) {}
/*  663 */         LuceneIndex.this.repair(e);
/*      */       } catch (AssertionError e) {
/*      */         try {
/*  666 */           writer.close(false);
/*      */         }
/*      */         catch (Throwable ignore) {}
/*  669 */         LuceneIndex.this.repair(e);
/*      */       } catch (IOException e) {
/*  671 */         ZimbraLog.index.error("Failed to merge IndexWriter", e);
/*      */       } finally {
/*  673 */         scheduler.release();
/*  674 */         this.ref.dec();
/*      */       }
/*      */     }
/*      */   }
/*      */   
/*      */   private IndexWriterConfig getWriterConfig() {
/*  680 */     IndexWriterConfig config = new IndexWriterConfig(VERSION, this.mailbox.index.getAnalyzer());
/*  681 */     config.setMergeScheduler(new MergeScheduler(null));
/*  682 */     config.setMaxBufferedDocs(LC.zimbra_index_lucene_max_buffered_docs.intValue());
/*  683 */     config.setRAMBufferSizeMB(LC.zimbra_index_lucene_ram_buffer_size_kb.intValue() / 1024.0D);
/*  684 */     if (LC.zimbra_index_lucene_merge_policy.booleanValue()) {
/*  685 */       LogDocMergePolicy policy = new LogDocMergePolicy();
/*  686 */       config.setMergePolicy(policy);
/*  687 */       policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue());
/*  688 */       policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue());
/*  689 */       policy.setMinMergeDocs(LC.zimbra_index_lucene_min_merge.intValue());
/*  690 */       if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) {
/*  691 */         policy.setMaxMergeDocs(LC.zimbra_index_lucene_max_merge.intValue());
/*      */       }
/*      */     } else {
/*  694 */       LogByteSizeMergePolicy policy = new LogByteSizeMergePolicy();
/*  695 */       config.setMergePolicy(policy);
/*  696 */       policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue());
/*  697 */       policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue());
/*  698 */       policy.setMinMergeMB(LC.zimbra_index_lucene_min_merge.intValue() / 1024.0D);
/*  699 */       if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) {
/*  700 */         policy.setMaxMergeMB(LC.zimbra_index_lucene_max_merge.intValue() / 1024.0D);
/*      */       }
/*      */     }
/*  703 */     return config;
/*      */   }
/*      */   
/*      */   public static ZimbraQueryResults search(ZimbraQuery zq) throws ServiceException {
/*  707 */     SearchParams params = zq.getParams();
/*  708 */     ZimbraLog.search.debug("query: %s", new Object[] { params.getQueryString() });
/*      */     
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*  717 */     boolean isTaskSort = false;
/*  718 */     boolean isLocalizedSort = false;
/*  719 */     SortBy originalSort = params.getSortBy();
/*  720 */     switch (originalSort) {
/*      */     case TASK_DUE_ASC: 
/*  722 */       isTaskSort = true;
/*  723 */       params.setSortBy(SortBy.DATE_DESC);
/*  724 */       break;
/*      */     case TASK_DUE_DESC: 
/*  726 */       isTaskSort = true;
/*  727 */       params.setSortBy(SortBy.DATE_DESC);
/*  728 */       break;
/*      */     case TASK_STATUS_ASC: 
/*  730 */       isTaskSort = true;
/*  731 */       params.setSortBy(SortBy.DATE_DESC);
/*  732 */       break;
/*      */     case TASK_STATUS_DESC: 
/*  734 */       isTaskSort = true;
/*  735 */       params.setSortBy(SortBy.DATE_DESC);
/*  736 */       break;
/*      */     case TASK_PERCENT_COMPLETE_ASC: 
/*  738 */       isTaskSort = true;
/*  739 */       params.setSortBy(SortBy.DATE_DESC);
/*  740 */       break;
/*      */     case TASK_PERCENT_COMPLETE_DESC: 
/*  742 */       isTaskSort = true;
/*  743 */       params.setSortBy(SortBy.DATE_DESC);
/*  744 */       break;
/*      */     case NAME_LOCALIZED_ASC: 
/*      */     case NAME_LOCALIZED_DESC: 
/*  747 */       isLocalizedSort = true;
/*      */     }
/*      */     
/*      */     
/*  751 */     ZimbraQueryResults results = zq.execute();
/*  752 */     if (isTaskSort) {
/*  753 */       results = new ReSortingQueryResults(results, originalSort, null);
/*      */     }
/*  755 */     if (isLocalizedSort) {
/*  756 */       results = new ReSortingQueryResults(results, originalSort, params);
/*      */     }
/*  758 */     return results;
/*      */   }
/*      */   
/*      */   public boolean isPendingDelete()
/*      */   {
/*  763 */     return this.pendingDelete.get();
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*  768 */   public void setPendingDelete(boolean pendingDelete) { this.pendingDelete.set(pendingDelete); }
/*      */   
/*      */   public void optimize() {}
/*      */   
/*      */   public static final class Factory implements IndexStore.Factory {
/*  773 */     public Factory() { BooleanQuery.setMaxClauseCount(LC.zimbra_index_lucene_max_terms_per_query.intValue()); }
/*      */     
/*      */     public LuceneIndex getIndexStore(Mailbox mbox)
/*      */       throws ServiceException
/*      */     {
/*  778 */       return new LuceneIndex(mbox, null);
/*      */     }
/*      */     
/*      */     public void destroy()
/*      */     {
/*  783 */       LuceneIndex.SEARCHER_CACHE.asMap().clear();
/*      */       
/*  785 */       for (LuceneIndex.IndexSearcherImpl searcher : LuceneIndex.GAL_SEARCHER_CACHE.values()) {
/*  786 */         Closeables.closeQuietly(searcher);
/*      */       }
/*  788 */       LuceneIndex.GAL_SEARCHER_CACHE.clear();
/*      */     }
/*      */   }
/*      */   
/*      */   private static final class LuceneIndexerImpl implements Indexer {
/*      */     private final LuceneIndex.IndexWriterRef writer;
/*      */     
/*      */     LuceneIndexerImpl(LuceneIndex.IndexWriterRef writer) {
/*  796 */       this.writer = writer;
/*      */     }
/*      */     
/*      */     public void close() throws IOException
/*      */     {
/*  801 */       this.writer.index.commitWriter();
/*  802 */       ZimbraIndexSearcher searcher = null;
/*  803 */       if (this.writer.getIndex().mailbox.isGalSyncMailbox()) {
/*  804 */         searcher = (ZimbraIndexSearcher)LuceneIndex.GAL_SEARCHER_CACHE.get(Integer.valueOf(this.writer.getIndex().mailbox.getId()));
/*      */       } else {
/*  806 */         searcher = (ZimbraIndexSearcher)LuceneIndex.SEARCHER_CACHE.getIfPresent(Integer.valueOf(this.writer.getIndex().mailbox.getId()));
/*      */       }
/*  808 */       if (searcher != null) {
/*  809 */         LuceneIndex.ZimbraLuceneIndexReader ndxReader = (LuceneIndex.ZimbraLuceneIndexReader)searcher.getIndexReader();
/*  810 */         IndexReader newReader = IndexReader.openIfChanged(ndxReader.getLuceneReader(), true);
/*  811 */         if (newReader != null) {
/*  812 */           if (this.writer.getIndex().mailbox.isGalSyncMailbox())
/*      */           {
/*  814 */             Closeables.closeQuietly((Closeable)LuceneIndex.GAL_SEARCHER_CACHE.put(Integer.valueOf(this.writer.getIndex().mailbox.getId()), new LuceneIndex.IndexSearcherImpl(newReader)));
/*      */ 
/*      */ 
/*      */           }
/*      */           else
/*      */           {
/*      */ 
/*  821 */             LuceneIndex.SEARCHER_CACHE.asMap().put(Integer.valueOf(this.writer.getIndex().mailbox.getId()), new LuceneIndex.IndexSearcherImpl(newReader));
/*      */           }
/*      */         }
/*      */       }
/*      */     }
/*      */     
/*      */ 
/*      */     public void compact()
/*      */     {
/*  830 */       LuceneIndex.MergeScheduler scheduler = (LuceneIndex.MergeScheduler)this.writer.get().getConfig().getMergeScheduler();
/*  831 */       scheduler.lock();
/*      */       try {
/*  833 */         ZimbraLog.index.info("Force merge deletes %d", new Object[] { Integer.valueOf(this.writer.get().maxDoc() - this.writer.get().numDocs()) });
/*  834 */         this.writer.get().forceMergeDeletes(true);
/*      */       } catch (IOException e) {
/*  836 */         ZimbraLog.index.error("Failed to optimize index", e);
/*      */       } finally {
/*  838 */         scheduler.release();
/*      */       }
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */     public synchronized int maxDocs()
/*      */     {
/*  846 */       return this.writer.get().maxDoc();
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */     public synchronized void addDocument(Folder folder, MailItem item, List<IndexDocument> docs)
/*      */       throws IOException
/*      */     {
/*  858 */       if ((docs == null) || (docs.isEmpty())) {
/*  859 */         return;
/*      */       }
/*      */       
/*      */ 
/*      */ 
/*  864 */       switch (LuceneIndex.4.$SwitchMap$com$zimbra$cs$mailbox$MailItem$IndexStatus[item.getIndexStatus().ordinal()]) {
/*      */       case 1: 
/*      */       case 2: 
/*  867 */         Term term = new Term("l.mbox_blob_id", String.valueOf(item.getId()));
/*  868 */         this.writer.get().deleteDocuments(term);
/*  869 */         break;
/*      */       case 3: 
/*      */         break;
/*      */       default: 
/*  873 */         if (!$assertionsDisabled) throw new AssertionError(item.getIndexId());
/*      */         break;
/*      */       }
/*  876 */       for (IndexDocument doc : docs)
/*      */       {
/*  878 */         synchronized (doc) {
/*  879 */           IndexStore.setFields(item, doc);
/*  880 */           Document luceneDoc = doc.toDocument();
/*  881 */           if (ZimbraLog.index.isTraceEnabled()) {
/*  882 */             ZimbraLog.index.trace("Adding lucene document %s", new Object[] { luceneDoc.toString() });
/*      */           }
/*  884 */           this.writer.get().addDocument(luceneDoc);
/*      */         }
/*      */       }
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */     public void deleteDocument(List<Integer> ids)
/*      */       throws IOException
/*      */     {
/*  898 */       for (Integer id : ids) {
/*  899 */         Term term = new Term("l.mbox_blob_id", id.toString());
/*  900 */         this.writer.get().deleteDocuments(term);
/*  901 */         ZimbraLog.index.debug("Deleted documents id=%d", new Object[] { id });
/*      */       }
/*      */     }
/*      */   }
/*      */   
/*      */ 
/*      */   private final class IndexWriterRef
/*      */   {
/*      */     private final LuceneIndex index;
/*      */     
/*      */     private final IndexWriter writer;
/*  912 */     private final AtomicInteger count = new AtomicInteger(1);
/*      */     
/*      */     IndexWriterRef(LuceneIndex index, IndexWriter writer) {
/*  915 */       this.index = index;
/*  916 */       this.writer = writer;
/*      */     }
/*      */     
/*      */     IndexWriter get() {
/*  920 */       return this.writer;
/*      */     }
/*      */     
/*      */     LuceneIndex getIndex() {
/*  924 */       return this.index;
/*      */     }
/*      */     
/*      */     void inc() {
/*  928 */       this.count.incrementAndGet();
/*      */     }
/*      */     
/*      */     void dec() {
/*  932 */       synchronized (this.index) {
/*  933 */         if (this.count.decrementAndGet() <= 0) {
/*  934 */           this.index.closeWriter();
/*      */         }
/*      */       }
/*      */     }
/*      */   }
/*      */   
/*      */ 
/*      */ 
/*      */   private static final class IndexSearcherImpl
/*      */     implements ZimbraIndexSearcher
/*      */   {
/*  945 */     private final AtomicInteger count = new AtomicInteger(1);
/*      */     private final IndexSearcher luceneSearcher;
/*      */     private final ZimbraIndexReader luceneReader;
/*      */     
/*      */     IndexSearcherImpl(IndexReader reader) {
/*  950 */       this.luceneSearcher = new IndexSearcher(reader);
/*  951 */       this.luceneReader = new LuceneIndex.ZimbraLuceneIndexReader(this.luceneSearcher.getIndexReader(), null);
/*      */     }
/*      */     
/*      */     void inc() {
/*  955 */       this.count.incrementAndGet();
/*      */     }
/*      */     
/*      */     public void close() throws IOException
/*      */     {
/*  960 */       if (this.count.decrementAndGet() == 0) {
/*  961 */         ZimbraLog.search.debug("Close IndexSearcher");
/*      */         try {
/*  963 */           Closeables.closeQuietly(this.luceneSearcher);
/*      */         } finally {
/*  965 */           Closeables.closeQuietly(getIndexReader());
/*  966 */           LuceneIndex.READER_THROTTLE.release();
/*      */         }
/*      */       }
/*      */     }
/*      */     
/*      */     public Document doc(ZimbraIndexDocumentID docID) throws IOException
/*      */     {
/*  973 */       if ((docID instanceof ZimbraLuceneDocumentID)) {
/*  974 */         ZimbraLuceneDocumentID zlDocID = (ZimbraLuceneDocumentID)docID;
/*  975 */         return this.luceneSearcher.doc(zlDocID.getLuceneDocID());
/*      */       }
/*  977 */       throw new IllegalArgumentException("Expected a ZimbraLuceneDocumentID");
/*      */     }
/*      */     
/*      */     public int docFreq(Term term) throws IOException
/*      */     {
/*  982 */       return this.luceneSearcher.docFreq(term);
/*      */     }
/*      */     
/*      */     public ZimbraIndexReader getIndexReader()
/*      */     {
/*  987 */       return this.luceneReader;
/*      */     }
/*      */     
/*      */     public ZimbraTopDocs search(Query query, int n) throws IOException
/*      */     {
/*  992 */       return ZimbraTopDocs.create(this.luceneSearcher.search(query, n));
/*      */     }
/*      */     
/*      */     public ZimbraTopDocs search(Query query, ZimbraTermsFilter filter, int n) throws IOException
/*      */     {
/*  997 */       TermsFilter luceneFilter = filter == null ? null : new TermsFilter(filter.getTerms());
/*  998 */       return ZimbraTopDocs.create(this.luceneSearcher.search(query, luceneFilter, n));
/*      */     }
/*      */     
/*      */     public ZimbraTopFieldDocs search(Query query, ZimbraTermsFilter filter, int n, Sort sort) throws IOException
/*      */     {
/* 1003 */       TermsFilter luceneFilter = filter == null ? null : new TermsFilter(filter.getTerms());
/* 1004 */       return ZimbraTopFieldDocs.create(this.luceneSearcher.search(query, luceneFilter, n, sort));
/*      */     }
/*      */   }
/*      */   
/*      */   public static final class ZimbraLuceneIndexReader implements ZimbraIndexReader
/*      */   {
/*      */     private final IndexReader luceneReader;
/*      */     
/*      */     private ZimbraLuceneIndexReader(IndexReader indexReader) {
/* 1013 */       this.luceneReader = indexReader;
/*      */     }
/*      */     
/*      */     public void close() throws IOException
/*      */     {
/* 1018 */       Closeables.closeQuietly(getLuceneReader());
/*      */     }
/*      */     
/*      */     public int numDocs()
/*      */     {
/* 1023 */       return getLuceneReader().numDocs();
/*      */     }
/*      */     
/*      */     public int numDeletedDocs()
/*      */     {
/* 1028 */       return getLuceneReader().numDeletedDocs();
/*      */     }
/*      */     
/*      */ 
/*      */ 
/*      */ 
/*      */ 
/*      */     public ZimbraIndexReader.TermFieldEnumeration getTermsForField(String field, String firstTermValue)
/*      */       throws IOException
/*      */     {
/* 1038 */       return new LuceneTermValueEnumeration(field, firstTermValue, null);
/*      */     }
/*      */     
/*      */     private final class LuceneTermValueEnumeration implements ZimbraIndexReader.TermFieldEnumeration {
/*      */       private TermEnum termEnumeration;
/*      */       private final String field;
/*      */       
/*      */       private LuceneTermValueEnumeration(String field, String firstTermValue) throws IOException {
/* 1046 */         this.termEnumeration = LuceneIndex.ZimbraLuceneIndexReader.this.getLuceneReader().terms(new Term(field, firstTermValue));
/* 1047 */         this.field = field;
/*      */       }
/*      */       
/*      */       public boolean hasMoreElements()
/*      */       {
/* 1052 */         if (this.termEnumeration == null) {
/* 1053 */           return false;
/*      */         }
/* 1055 */         Term term = this.termEnumeration.term();
/* 1056 */         return (term != null) && (this.field.equals(term.field()));
/*      */       }
/*      */       
/*      */       public BrowseTerm nextElement()
/*      */       {
/* 1061 */         if (this.termEnumeration == null) {
/* 1062 */           throw new NoSuchElementException("No more values");
/*      */         }
/* 1064 */         Term term = this.termEnumeration.term();
/* 1065 */         if ((term != null) && (this.field.equals(term.field()))) {
/* 1066 */           BrowseTerm nextVal = new BrowseTerm(term.text(), this.termEnumeration.docFreq());
/*      */           try {
/* 1068 */             this.termEnumeration.next();
/*      */           } catch (IOException e) {
/* 1070 */             Closeables.closeQuietly(this.termEnumeration);
/* 1071 */             this.termEnumeration = null;
/*      */           }
/* 1073 */           return nextVal;
/*      */         }
/* 1075 */         Closeables.closeQuietly(this.termEnumeration);
/* 1076 */         throw new NoSuchElementException("No more values");
/*      */       }
/*      */       
/*      */       public void close()
/*      */         throws IOException
/*      */       {
/* 1082 */         if (this.termEnumeration != null) {
/* 1083 */           Closeables.closeQuietly(this.termEnumeration);
/*      */         }
/* 1085 */         this.termEnumeration = null;
/*      */       }
/*      */     }
/*      */     
/*      */     public IndexReader getLuceneReader() {
/* 1090 */       return this.luceneReader;
/*      */     }
/*      */   }
/*      */ }


/* Location:              /home/mint/zimbrastore.jar!/com/zimbra/cs/index/LuceneIndex.class
 * Java compiler version: 7 (51.0)
 * JD-Core Version:       0.7.1
 */